In [191]:
%matplotlib notebook
import itertools
import logging
from functools import partial
from collections import defaultdict, OrderedDict
import gensim
import matplotlib
from matplotlib import rc
import matplotlib.pyplot as plt
from matplotlib import colors as plt_colors
import numpy as np
import pandas as pnd
import os
from sklearn.cluster import *
from sklearn.preprocessing import normalize
from sklearn.cross_validation import train_test_split
from sklearn.decomposition import PCA, RandomizedPCA
from sklearn.manifold import TSNE
from sklearn import svm, metrics
from multiprocessing import Pool
from knub.thesis.util import *
pnd.set_option("display.max_colwidth", 200)
LIGHT_COLORS = ["#a6cee3", "#b2df8a", "#fb9a99", "#fdbf6f", "#cab2d6", "#ffff99"] # light colors
DARK_COLORS = ["#1f78b4", "#33a02c", "#e31a1c", "#ff7f00", "#6a3d9a"] # dark colors
rc('font', **{'family': 'serif', 'serif': ['Computer Modern']})
rc('text', usetex=True)
SIZE = 11
rc('font', size=SIZE) # controls default text sizes
rc('axes', titlesize=SIZE) # fontsize of the axes title
rc('axes', labelsize=SIZE) # fontsize of the x and y labels
rc('xtick', labelsize=SIZE) # fontsize of the tick labels
rc('ytick', labelsize=SIZE) # fontsize of the tick labels
rc('legend', fontsize=SIZE) # legend fontsize
rc('figure', titlesize=SIZE) # fontsize of the figure title
def cm2inch(*tupl):
inch = 2.54
if isinstance(tupl[0], tuple):
return tuple(i/inch for i in tupl[0])
else:
return tuple(i/inch for i in tupl)
In [91]:
def evaluate_single(df_param):
nr_inclusions = len(df_param)
nr_succ = len(df_param[df_param.successful])
return float(nr_succ) / nr_inclusions
def evaluate_df(df_param, print_eval):
for method, df_group in df_param.groupby("method"):
if "mixture" in method:
continue
succ_prob = evaluate_single(df_group)
if print_eval:
print "%s: %.2f" % (method, succ_prob)
def evaluate_file(f, print_eval=True):
df_file = pnd.read_csv("/home/knub/Repositories/master-thesis/webapp/out/" + f, sep="\t", header=None)
df_file.columns = ["inclusion_idx", "inclusion_id", "method", "words", "intruder", "selected_word", "successful"]
evaluate_df(df_file, print_eval)
return df_file
In [92]:
for f_name in sorted([f for f in os.listdir("../webapp/out") if f.endswith(".txt")]):
print f_name
evaluate_file(f_name)
print
Evaluating all results
In [100]:
dfs = []
for f_name in sorted([f for f in os.listdir("../webapp/out") if f.endswith(".txt")]):
df_tmp = evaluate_file(f_name, print_eval=False)
df_tmp["evaluator"] = f_name.replace(".txt", "").replace("20_", "")
dfs.append(df_tmp)
df_all = pnd.concat(dfs)
evaluate_df(df_all, print_eval=True)
In [230]:
def aggregate_samples(df_param):
return all(df_param)
df_agreement = df_all.groupby(["inclusion_id", "method"], as_index=False)["successful"].agg(aggregate_samples).reset_index()
pnd.reset_option('display.max_rows')
df_agreement.groupby("method")["successful"].agg(["mean", "count"])
Out[230]:
In [98]:
df_agreement.groupby(["method"])["mean"].mean()
Out[98]:
In [241]:
def build_word_intrusion(df_param):
models = sorted(list(set(df_param.method)))
evaluators = sorted(list(set(df_param.evaluator)))
df_return = pnd.DataFrame(columns=models, index=evaluators)
df_return.index.name = "evaluator"
df_return.columns.name = "model"
print models
for evaluator, df_group_evaluator in df_param.groupby("evaluator"):
for model, df_group_model in df_group_evaluator.groupby("method"):
df_return.set_value(evaluator, model, evaluate_single(df_group_model))
return df_return
def plot_word_intrusion(df_param):
df_param = pnd.melt(df_param.reset_index(), id_vars=['evaluator'])
df_param = df_param.dropna()
df_param = df_param.sort_values(["model", "value"])
models = sorted(list(set(df_param["model"])))
for m in models:
df_param.loc[df_param["model"] == m, "offset"] = range(len(df_param["model"] == m))
models = OrderedDict([(v, k+1) for k, v in enumerate(models)])
scatters = np.linspace(-0.25, 0.25, 8)
df_param["scatter_x"] = df_param.apply(lambda x: models[x.model] + scatters[x.offset], axis=1)
plt.figure(figsize=cm2inch(12, 6), dpi=300)
plt.xlim(0.6, 4.8)
plt.ylim((0.,1.0))
plt.xticks(models.values(), ["LDA", "LFTM", "TopicVec", "WELDA"], rotation='horizontal')
plt.scatter(df_param["scatter_x"], df_param["value"], c="black", marker='x', s=20, alpha=1.0, lw = 0.5)
width = 0.25
for m in models:
mean = df_param[df_param.model == m]["value"].mean()
#plt.axhline(y=mean)
#print models[m]
plt.hlines(y=mean, xmin=models[m]-width, xmax=models[m]+width, linestyles="dashed", linewidth=0.5)
plt.text(models[m] + width + 0.03, mean, "$%.2f$" % mean, verticalalignment="center")
df_no_mixture = df_all[df_all.method.apply(lambda x: "mixture" not in x)]
df_20 = df_no_mixture[df_no_mixture.method.apply(lambda x: "20" in x)]
df_50 = df_no_mixture[df_no_mixture.method.apply(lambda x: "20" not in x)]
df_word_intrusion_20 = build_word_intrusion(df_20)
plot_word_intrusion(df_word_intrusion_20)
df_word_intrusion_50 = build_word_intrusion(df_50)
plot_word_intrusion(df_word_intrusion_50)
In [209]:
df_word_intrusion_20
Out[209]:
In [210]:
df_word_intrusion_50
Out[210]: